# coding:utf-8

'''
  This is the engine of the spider framework.
  It's better to deal with the whole process and some emergency.
'''
import threading

from src import settings
from src.utils import urls
from src.utils.SingletomRedisPool import RedisPool
from src.core import datafilter


def start_url():
    """
        add the start_url in settings.py into url_list after filter
    """
    from . import task_manager
    p = threading.Thread(target=task_manager.run)
    p.start()
    [add_url(url) for url in settings.start_url]


def add_url(url):
    try:
        if not urls.isExist(url):
            conn = RedisPool.getConn()
            conn.lpush(settings.REDIS_QUEUE_URLS, url)
    except:
        print("REDIS CONNECTION EXCEPT!")


def auto_deal_response(response):
    response.encoding = response.apparent_encoding
    print("OK:{}<{}>".format(response.url, response.status_code))
    deal_reponse(response)


def deal_reponse(response):
    # writed by user
    response.encoding="gbk"
    datafilter.store(response.text,response.url)
    pass
